{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.10 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "d719d340c32571272b56ad77079bbfc0411b1da5f2b0ea2e59bff1425b9bbb3e"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spider import Spider"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_url = r'https://detail.1688.com/offer/563971778509.html'\n",
    "browser_user_agent = r\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.36\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "error",
     "ename": "RuntimeError",
     "evalue": "Cannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead.",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-048eaed1c272>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mspider\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSpider\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspider\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_url\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'user-agent'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbrowser_user_agent\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'referer'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'https://detail.1688.com/'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrender\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/Developer/forestProject/spider_1688/spider.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(url, render, headers, allow_redirects)\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m200\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mrender\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhtml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/anaconda3/envs/spider_1688/lib/python3.7/site-packages/requests_html.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, retries, script, wait, scrolldown, sleep, reload, timeout, keep_page)\u001b[0m\n\u001b[1;32m    584\u001b[0m         \"\"\"\n\u001b[1;32m    585\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbrowser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbrowser\u001b[0m  \u001b[0;31m# Automatically create a event loop and browser\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m         \u001b[0mcontent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/anaconda3/envs/spider_1688/lib/python3.7/site-packages/requests_html.py\u001b[0m in \u001b[0;36mbrowser\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    727\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_event_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    728\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 729\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Cannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    730\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_browser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbrowser\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    731\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_browser\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Cannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead."
     ]
    }
   ],
   "source": [
    "spider = Spider()\n",
    "response = spider.request(test_url, headers={'user-agent':browser_user_agent,'referer': 'https://detail.1688.com/'}, render=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'\\r\\n\\r\\n\\r\\n<html>\\n\\n<head>\\n    <meta charset=\"utf-8\">\\n    <meta name=\"viewport\" content=\"width=device-width,initial-scale=1,minimum-scale=1,maximum-scale=1,user-scalable=no\" />\\n    <script src=\"https://g.alicdn.com/mtb/lib-flexible/0.3.2/flexible.js\" crossorigin></script>\\n  <script src=\"//g.alicdn.com/code/lib/qrcodejs/1.0.0/qrcode.min.js\" crossorigin></script>\\n          <link rel=\"stylesheet\" href=\"https://g.alicdn.com/bsop-static/sufei-punish/0.0.104/build/main.css\">\\n\\n            </head>\\n\\n<body><script>\\nwith(document)with(body)with(insertBefore(createElement(\"script\"),firstChild))setAttribute(\"exparams\",\"category=&userid=&aplus&asid=AQAAAABwSFNgqCSscQAAAADG59vI9xRleA==&aat=&abb=&c_signed=&hn=laputa011087187063%2ecenter%2ena62&at_bu=cbu&sidx=cOQD/RXiniovyXr/nEQA9pJ7kFNwnPecWWf3liIib+eG2Q5qkhRIEL0isDzm+RS5ylaj40NqReC859lVVEiBFDX7fRBlaARPVQr0svYsyzWf3PubIW0GQZHQOthfWZ6zHLiG86OnB5n7i5VeUX5Fk1j3uFRCfBBlbgv0zQkB0ts=\",id=\"beacon-aplus\",src=\"//g.alicdn.com/alilog/??/aplus_plugin_b2bfront/index.js,mlog/aplus_v2.js\")\\n</script><script>\\nwith(document)with(body)with(insertBefore(createElement(\"script\"),firstChild))setAttribute(\"exparams\",\"category=&userid=&aplus&yunid=&&trid=212a938616160707680212053ec1cc&asid=AQAAAABwSFNgghGJVQAAAADw8CTd9whyFw==\",id=\"tb-beacon-aplus\",src=(location>\"https\"?\"//g\":\"//g\")+\".alicdn.com/alilog/mlog/aplus_v2.js\")\\n</script>\\n\\n    <punish-component />\\n    <script>\\n        window._config_ = {\\n            \"renderTo\": \"#nocaptcha\",\\n            \"NCTOKENSTR\": \"d50b4c944bc3d4cba9d9ae9fbab58e73\",\\n            \"action\": \"captcha\",\\n            \"HOST\": \"detail.1688.com:443\",\\n\\t\\t\\t\"PATH\": \"/offer/638463438028.html\",\\n            \"FORMACTIOIN\": \"/_____tmd_____/verify/\",\\n            \"BXSTEP\": \"100\",\\n            \"SECDATA\": \"5e0c8e1365474455070961b803bd560607b52cabf5960afff39b64ce58073f78f0e20b9895bda9ee34121f0880346f73e7785116f155e519e51898407f8bfbfd136dd0ffd5c57ac2657a76eb362de78c7bf3fd8381b0cf846dca6234fbaf10bcbb59732cfb08afd44561a0c048d1d4c2b1726de5abc460619c989eed3429b470d472bd4d030986c5015c5b92e2094a7c9302c9cf60ede011a718a5f78bdd23dd1bb63220415ffa9edf79d0f7f2b9bd4ca8751f963303ddca91559e3c528e4bdcbe4badbcebe3003e8253ab016f39d050d586e748c2b53fbd5274b12ca5527a868f3dfa032cd549b84cfb51d1567a9a117b32b8c86378740ae2b1f581a633c696ad921bb1d36afdc5973c0455682491a957f7918a4f2572499cc398910575bb4a5891b3b86c5643461bd303081a465e61acd0f057be032c18329d0ee778d0f284ac62b8c07ac9734d6799e4f8b11e11a59dab80f89293688707c3aec5d53b80edeecaaf895e5a57492d821cbccdbc4384d1726e2cd0ab007dbfc6a9d71f03c56656a0b84971ee4d79e7ac6e2d28c4fbd8ab570034bcc3f82cda2fb36536ba0f787925581aa525a19044986302a06b05a785bd512ca59d5f0a1f3a05fc7032701a179da721df30249b85829bd5d67b802f\",\\n            \"NCAPPKEY\": \"X82Y__0e1748d45e9398849fd9ba9e6eb0c705\"\\n        }\\n    </script>\\n</body>\\n <script src=\"//g.alicdn.com/bsop-static/sufei-punish/0.0.104/build/punishpage.min.js\" crossorigin></script>\\n<script type=\"text/javascript\"\\n    src=\"https://g.alicdn.com/dt/tracker/4.0.0/??tracker.Tracker.js,tracker.interfaceTrackerPlugin.js,tracker.performanceTrackerPlugin.js\"\\n    crossorigin></script>\\n\\n  <script type=\"text/javascript\">\\n    var tracker = new window.Tracker({uidResolver: function() {\\n    // �����ȡ userId �\\u07fc�����ʵ��\\n  return \"d50b4c944bc3d4cba9d9ae9fbab58e73__1.1.1\";\\n  },  pid: \\'punish-page\\', plugins: [[window.interfaceTrackerPlugin], [window.performanceTrackerPlugin, { sampleRate: 1 }]] });\\n    tracker.install();\\n    tracker.log({\\n      code: 11,  // ϵͳ�Զ����ɣ������\\u07b8� 100%\\n      c1: \\'d50b4c944bc3d4cba9d9ae9fbab58e73\\',  \\n      msg: \\'����ͨ�� web ҳ��-��֤��\\',  // �쳣��Ϣ���Ƽ�����\\n      sampleRate: 1.00,  // Ŀǰ������Ϊ 100.00%\\n    });\\n  </script>\\n\\n</html>'"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "response.html.html\n",
    "# response.html.find('#mod-detail-attributes .obj-content tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}